import scrapy
import json
from mySpider.items import JingDongComment, Comment


class jingdongSpider(scrapy.Spider):
    name = "jingdongSpider"

    # 爬虫入口
    def start_requests(self):
        # 请求头，模拟浏览器访问
        head = {
            "Referer": "https://item.jd.com/100006729770.html",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"
        }

        base_url = "https://sclub.jd.com/comment/productPageComments.action?productId=100006729770&score=0&sortType=5&page=%s&pageSize=10&isShadowSku=0&fold=1"


        for page in range(100):
            url = base_url % page
            # 发起一个http请求
            yield scrapy.Request(url=url, callback=self.parse, headers=head)

    # response  响应
    def parse(self, response):
        # 获取网页内容
        print(response.text)
        # 将json字符串转换成json对象
        json_str = json.loads(response.text)

        # 获取所有评价
        comments = json_str["comments"]

        for comment in comments:
            # 获取评价内容
            content = str(comment["content"]).replace("\n", " ")
            id = comment["id"]
            nickname = comment["nickname"]
            productColor = comment["productColor"]
            productSize = comment["productSize"]
            userLevelName = comment["userLevelName"]
            userClientShow = comment["userClientShow"]

            line = "%s\t%s\t%s\t%s\t%s\t%s\t%s" % (
                id, userClientShow, userLevelName, nickname, content, productColor, productSize)

            # 构建自定义类的对象
            item = Comment()
            item["line"] = line

            # 将item发生到pipeline
            yield item
